{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle/input/fake-news-machine-hack/Test.csv\n",
      "/kaggle/input/fake-news-machine-hack/Train.csv\n",
      "/kaggle/input/fake-news-machine-hack/sample submission.csv\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the read-only \"../input/\" directory\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "\n",
    "# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n",
    "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv(\"/kaggle/input/fake-news-machine-hack/Train.csv\")\n",
    "test = pd.read_csv(\"/kaggle/input/fake-news-machine-hack/Test.csv\")\n",
    "samp = pd.read_csv(\"/kaggle/input/fake-news-machine-hack/sample submission.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Labels</th>\n",
       "      <th>Text</th>\n",
       "      <th>Text_Tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Says the Annies List political group supports ...</td>\n",
       "      <td>abortion</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>When did the decline of coal start? It started...</td>\n",
       "      <td>energy,history,job-accomplishments</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Hillary Clinton agrees with John McCain \"by vo...</td>\n",
       "      <td>foreign-policy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>Health care reform legislation is likely to ma...</td>\n",
       "      <td>health-care</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>The economic turnaround started at the end of ...</td>\n",
       "      <td>economy,jobs</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Labels                                               Text  \\\n",
       "0       1  Says the Annies List political group supports ...   \n",
       "1       2  When did the decline of coal start? It started...   \n",
       "2       3  Hillary Clinton agrees with John McCain \"by vo...   \n",
       "3       1  Health care reform legislation is likely to ma...   \n",
       "4       2  The economic turnaround started at the end of ...   \n",
       "\n",
       "                             Text_Tag  \n",
       "0                            abortion  \n",
       "1  energy,history,job-accomplishments  \n",
       "2                      foreign-policy  \n",
       "3                         health-care  \n",
       "4                        economy,jobs  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['Text'] = train['Text'] + ' ' +  train['Text_Tag'].fillna(\"\").apply(lambda x : ' '.join(x.split(',')))\n",
    "test['Text'] = test['Text'] + ' ' +  test['Text_Tag'].fillna(\"\").apply(lambda x : ' '.join(x.split(',')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "target = 'Labels'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import plotly\n",
    "from catboost import CatBoostClassifier\n",
    "from lightgbm import LGBMClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import KFold\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 553
    },
    "colab_type": "code",
    "id": "_DNyZd06MYka",
    "outputId": "4af6bc7d-e1d6-4906-dfaf-a8a23b6c6d31"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Labels</th>\n",
       "      <th>Text</th>\n",
       "      <th>Text_Tag</th>\n",
       "      <th>description_word_len</th>\n",
       "      <th>mean</th>\n",
       "      <th>max</th>\n",
       "      <th>min</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Says the Annies List political group supports ...</td>\n",
       "      <td>abortion</td>\n",
       "      <td>13</td>\n",
       "      <td>17.778846</td>\n",
       "      <td>46.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.0</td>\n",
       "      <td>When did the decline of coal start? It started...</td>\n",
       "      <td>energy,history,job-accomplishments</td>\n",
       "      <td>28</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>28.0</td>\n",
       "      <td>28.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.0</td>\n",
       "      <td>Hillary Clinton agrees with John McCain \"by vo...</td>\n",
       "      <td>foreign-policy</td>\n",
       "      <td>21</td>\n",
       "      <td>24.240000</td>\n",
       "      <td>515.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Health care reform legislation is likely to ma...</td>\n",
       "      <td>health-care</td>\n",
       "      <td>14</td>\n",
       "      <td>20.617577</td>\n",
       "      <td>56.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.0</td>\n",
       "      <td>The economic turnaround started at the end of ...</td>\n",
       "      <td>economy,jobs</td>\n",
       "      <td>12</td>\n",
       "      <td>19.948529</td>\n",
       "      <td>41.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11502</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Says his budget provides the highest state fun...</td>\n",
       "      <td>education</td>\n",
       "      <td>14</td>\n",
       "      <td>18.779026</td>\n",
       "      <td>52.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11503</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Ive been here almost every day. civil-rights c...</td>\n",
       "      <td>civil-rights,crime,criminal-justice</td>\n",
       "      <td>11</td>\n",
       "      <td>17.000000</td>\n",
       "      <td>22.0</td>\n",
       "      <td>11.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11504</th>\n",
       "      <td>NaN</td>\n",
       "      <td>In the early 1980s, Sen. Edward Kennedy secret...</td>\n",
       "      <td>bipartisanship,congress,foreign-policy,history</td>\n",
       "      <td>26</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>26.0</td>\n",
       "      <td>26.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11505</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Says an EPA permit languished under Strickland...</td>\n",
       "      <td>environment,government-efficiency</td>\n",
       "      <td>21</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>21.0</td>\n",
       "      <td>21.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11506</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Says the governor is going around the state ta...</td>\n",
       "      <td>state-budget,state-finances,taxes</td>\n",
       "      <td>33</td>\n",
       "      <td>23.909091</td>\n",
       "      <td>38.0</td>\n",
       "      <td>13.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11507 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Labels                                               Text  \\\n",
       "0         1.0  Says the Annies List political group supports ...   \n",
       "1         2.0  When did the decline of coal start? It started...   \n",
       "2         3.0  Hillary Clinton agrees with John McCain \"by vo...   \n",
       "3         1.0  Health care reform legislation is likely to ma...   \n",
       "4         2.0  The economic turnaround started at the end of ...   \n",
       "...       ...                                                ...   \n",
       "11502     NaN  Says his budget provides the highest state fun...   \n",
       "11503     NaN  Ive been here almost every day. civil-rights c...   \n",
       "11504     NaN  In the early 1980s, Sen. Edward Kennedy secret...   \n",
       "11505     NaN  Says an EPA permit languished under Strickland...   \n",
       "11506     NaN  Says the governor is going around the state ta...   \n",
       "\n",
       "                                             Text_Tag  description_word_len  \\\n",
       "0                                            abortion                    13   \n",
       "1                  energy,history,job-accomplishments                    28   \n",
       "2                                      foreign-policy                    21   \n",
       "3                                         health-care                    14   \n",
       "4                                        economy,jobs                    12   \n",
       "...                                               ...                   ...   \n",
       "11502                                       education                    14   \n",
       "11503             civil-rights,crime,criminal-justice                    11   \n",
       "11504  bipartisanship,congress,foreign-policy,history                    26   \n",
       "11505               environment,government-efficiency                    21   \n",
       "11506               state-budget,state-finances,taxes                    33   \n",
       "\n",
       "            mean    max   min  \n",
       "0      17.778846   46.0   3.0  \n",
       "1      28.000000   28.0  28.0  \n",
       "2      24.240000  515.0   6.0  \n",
       "3      20.617577   56.0   6.0  \n",
       "4      19.948529   41.0   7.0  \n",
       "...          ...    ...   ...  \n",
       "11502  18.779026   52.0   4.0  \n",
       "11503  17.000000   22.0  11.0  \n",
       "11504  26.000000   26.0  26.0  \n",
       "11505  21.000000   21.0  21.0  \n",
       "11506  23.909091   38.0  13.0  \n",
       "\n",
       "[11507 rows x 7 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "def clean(x):\n",
    "  x = re.sub(r'@\\w+',\" \",x)\n",
    "  x = re.sub(r'[^a-zA-Z]',\" \",x)\n",
    "  x = re.sub(r' [a-zA-Z]{1} ',\" \",x)\n",
    "  return x\n",
    "\n",
    "merge = pd.concat([train,test]).reset_index(drop=True)\n",
    "merge[\"description_word_len\"] = merge.apply(lambda x:len(re.findall(r\"\\w+\",x['Text'])),axis=1)\n",
    "m = merge.groupby('Text_Tag')['description_word_len'].agg(['mean','max',\"min\"])\n",
    "merge = merge.merge(m,on='Text_Tag',how=\"left\")\n",
    "# merge[\"Product_Description\"] = merge.apply(lambda x:clean(x['Product_Description']),axis=1)\n",
    "\n",
    "merge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 776
    },
    "colab_type": "code",
    "id": "o2f75qgHoktq",
    "outputId": "aba5eafd-e4c6-4f14-a472-659d28f68c2d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: transformers in /opt/conda/lib/python3.7/site-packages (2.11.0)\n",
      "Requirement already satisfied: tokenizers==0.7.0 in /opt/conda/lib/python3.7/site-packages (from transformers) (0.7.0)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.7/site-packages (from transformers) (3.0.10)\n",
      "Requirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from transformers) (1.18.5)\n",
      "Requirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.7/site-packages (from transformers) (4.45.0)\n",
      "Requirement already satisfied: packaging in /opt/conda/lib/python3.7/site-packages (from transformers) (20.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.7/site-packages (from transformers) (2020.4.4)\n",
      "Requirement already satisfied: sentencepiece in /opt/conda/lib/python3.7/site-packages (from transformers) (0.1.91)\n",
      "Requirement already satisfied: sacremoses in /opt/conda/lib/python3.7/site-packages (from transformers) (0.0.43)\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.7/site-packages (from transformers) (2.23.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /opt/conda/lib/python3.7/site-packages (from packaging->transformers) (2.4.7)\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.7/site-packages (from packaging->transformers) (1.14.0)\n",
      "Requirement already satisfied: click in /opt/conda/lib/python3.7/site-packages (from sacremoses->transformers) (7.1.1)\n",
      "Requirement already satisfied: joblib in /opt/conda/lib/python3.7/site-packages (from sacremoses->transformers) (0.14.1)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (3.0.4)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (2.9)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (1.24.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (2020.6.20)\n",
      "\u001b[33mWARNING: You are using pip version 20.2.2; however, version 20.2.3 is available.\n",
      "You should consider upgrading via the '/opt/conda/bin/python3.7 -m pip install --upgrade pip' command.\u001b[0m\n",
      "Collecting sentence-transformers\n",
      "  Downloading sentence-transformers-0.3.6.tar.gz (62 kB)\n",
      "\u001b[K     |████████████████████████████████| 62 kB 265 kB/s eta 0:00:011\n",
      "\u001b[?25hCollecting transformers<3.2.0,>=3.1.0\n",
      "  Downloading transformers-3.1.0-py3-none-any.whl (884 kB)\n",
      "\u001b[K     |████████████████████████████████| 884 kB 960 kB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied, skipping upgrade: tqdm in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (4.45.0)\n",
      "Requirement already satisfied, skipping upgrade: torch>=1.2.0 in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (1.5.1)\n",
      "Requirement already satisfied, skipping upgrade: numpy in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (1.18.5)\n",
      "Requirement already satisfied, skipping upgrade: scikit-learn in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (0.23.2)\n",
      "Requirement already satisfied, skipping upgrade: scipy in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (1.4.1)\n",
      "Requirement already satisfied, skipping upgrade: nltk in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (3.2.4)\n",
      "Requirement already satisfied, skipping upgrade: packaging in /opt/conda/lib/python3.7/site-packages (from transformers<3.2.0,>=3.1.0->sentence-transformers) (20.1)\n",
      "Collecting tokenizers==0.8.1.rc2\n",
      "  Downloading tokenizers-0.8.1rc2-cp37-cp37m-manylinux1_x86_64.whl (3.0 MB)\n",
      "\u001b[K     |████████████████████████████████| 3.0 MB 3.5 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied, skipping upgrade: regex!=2019.12.17 in /opt/conda/lib/python3.7/site-packages (from transformers<3.2.0,>=3.1.0->sentence-transformers) (2020.4.4)\n",
      "Requirement already satisfied, skipping upgrade: sentencepiece!=0.1.92 in /opt/conda/lib/python3.7/site-packages (from transformers<3.2.0,>=3.1.0->sentence-transformers) (0.1.91)\n",
      "Requirement already satisfied, skipping upgrade: sacremoses in /opt/conda/lib/python3.7/site-packages (from transformers<3.2.0,>=3.1.0->sentence-transformers) (0.0.43)\n",
      "Requirement already satisfied, skipping upgrade: filelock in /opt/conda/lib/python3.7/site-packages (from transformers<3.2.0,>=3.1.0->sentence-transformers) (3.0.10)\n",
      "Requirement already satisfied, skipping upgrade: requests in /opt/conda/lib/python3.7/site-packages (from transformers<3.2.0,>=3.1.0->sentence-transformers) (2.23.0)\n",
      "Requirement already satisfied, skipping upgrade: future in /opt/conda/lib/python3.7/site-packages (from torch>=1.2.0->sentence-transformers) (0.18.2)\n",
      "Requirement already satisfied, skipping upgrade: joblib>=0.11 in /opt/conda/lib/python3.7/site-packages (from scikit-learn->sentence-transformers) (0.14.1)\n",
      "Requirement already satisfied, skipping upgrade: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from scikit-learn->sentence-transformers) (2.1.0)\n",
      "Requirement already satisfied, skipping upgrade: six in /opt/conda/lib/python3.7/site-packages (from nltk->sentence-transformers) (1.14.0)\n",
      "Requirement already satisfied, skipping upgrade: pyparsing>=2.0.2 in /opt/conda/lib/python3.7/site-packages (from packaging->transformers<3.2.0,>=3.1.0->sentence-transformers) (2.4.7)\n",
      "Requirement already satisfied, skipping upgrade: click in /opt/conda/lib/python3.7/site-packages (from sacremoses->transformers<3.2.0,>=3.1.0->sentence-transformers) (7.1.1)\n",
      "Requirement already satisfied, skipping upgrade: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests->transformers<3.2.0,>=3.1.0->sentence-transformers) (1.24.3)\n",
      "Requirement already satisfied, skipping upgrade: chardet<4,>=3.0.2 in /opt/conda/lib/python3.7/site-packages (from requests->transformers<3.2.0,>=3.1.0->sentence-transformers) (3.0.4)\n",
      "Requirement already satisfied, skipping upgrade: idna<3,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests->transformers<3.2.0,>=3.1.0->sentence-transformers) (2.9)\n",
      "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests->transformers<3.2.0,>=3.1.0->sentence-transformers) (2020.6.20)\n",
      "Building wheels for collected packages: sentence-transformers\n",
      "  Building wheel for sentence-transformers (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for sentence-transformers: filename=sentence_transformers-0.3.6-py3-none-any.whl size=101181 sha256=ce3694ad1bc76f2d4e71a1ddc13141d53a1d9f165b8fadcf43c7839556915f3c\n",
      "  Stored in directory: /root/.cache/pip/wheels/b5/94/b4/953a1fd26652702c88112a188346df4cead56f0e3971a6d653\n",
      "Successfully built sentence-transformers\n",
      "Installing collected packages: tokenizers, transformers, sentence-transformers\n",
      "  Attempting uninstall: tokenizers\n",
      "    Found existing installation: tokenizers 0.7.0\n",
      "    Uninstalling tokenizers-0.7.0:\n",
      "      Successfully uninstalled tokenizers-0.7.0\n",
      "  Attempting uninstall: transformers\n",
      "    Found existing installation: transformers 2.11.0\n",
      "    Uninstalling transformers-2.11.0:\n",
      "      Successfully uninstalled transformers-2.11.0\n",
      "\u001b[31mERROR: After October 2020 you may experience errors when installing or updating packages. This is because pip will change the way that it resolves dependency conflicts.\n",
      "\n",
      "We recommend you use --use-feature=2020-resolver to test your packages with the new resolver before it becomes the default.\n",
      "\n",
      "allennlp 1.0.0 requires transformers<2.12,>=2.9, but you'll have transformers 3.1.0 which is incompatible.\u001b[0m\n",
      "Successfully installed sentence-transformers-0.3.6 tokenizers-0.8.1rc2 transformers-3.1.0\n",
      "\u001b[33mWARNING: You are using pip version 20.2.2; however, version 20.2.3 is available.\n",
      "You should consider upgrading via the '/opt/conda/bin/python3.7 -m pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers\n",
    "!pip install -U sentence-transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "OBlwfcOqdMZu",
    "outputId": "21da43fc-42ba-4357-95a4-6ad4cb84fd54"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TF version 2.3.0\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow.keras.backend as K\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "import tokenizers\n",
    "print('TF version',tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "JD11uNTGojEJ"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m W&B installed but not logged in.  Run `wandb login` or set the WANDB_API_KEY env variable.\n",
      "100%|██████████| 1.31G/1.31G [00:31<00:00, 41.6MB/s]\n"
     ]
    }
   ],
   "source": [
    "from sentence_transformers import SentenceTransformer\n",
    "sentence_embedder = SentenceTransformer('roberta-large-nli-stsb-mean-tokens')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 99,
     "referenced_widgets": [
      "b2bfcd4ac76e495ca31064bc7876041b",
      "425cd92f3c904a8ab278ab5863e1695a",
      "93ce70d6ec9340bcacf2c8ff2135ae72",
      "b4e18b0b524c47caaa443b5cb34fea05",
      "3dc97c159e9a4469b0e1c34ed921b54b",
      "b19b6b2e855241c59d2eea43329bf69b",
      "efd9f88490e34a71a43214449baace68",
      "2c0cb5298b4343168fc4f047125903af"
     ]
    },
    "colab_type": "code",
    "id": "N4FC41oqrtAU",
    "outputId": "b1b75191-26e0-429a-82e4-ba574fa46737"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9bb6169ef4e643a98eec52bda1d30536",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Batches', max=180.0, style=ProgressStyle(description_widt…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "CPU times: user 32.5 s, sys: 14.3 s, total: 46.8 s\n",
      "Wall time: 50.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "sentence_embeddings = sentence_embedder.encode(merge.Text.values.tolist(),batch_size=64,show_progress_bar=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zfqIxLgBsI3L"
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics  import log_loss\n",
    "from lightgbm import LGBMClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from catboost import CatBoostClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 244
    },
    "colab_type": "code",
    "id": "3sBFzaMFtxaJ",
    "outputId": "cda808fc-7989-4c00-8969-4e4f5ca2ee09"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>1020</th>\n",
       "      <th>1021</th>\n",
       "      <th>1022</th>\n",
       "      <th>1023</th>\n",
       "      <th>Text_Tag</th>\n",
       "      <th>Labels</th>\n",
       "      <th>description_word_len</th>\n",
       "      <th>mean</th>\n",
       "      <th>max</th>\n",
       "      <th>min</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.819478</td>\n",
       "      <td>0.553745</td>\n",
       "      <td>0.281095</td>\n",
       "      <td>0.483959</td>\n",
       "      <td>0.425502</td>\n",
       "      <td>0.377212</td>\n",
       "      <td>1.252342</td>\n",
       "      <td>-0.328106</td>\n",
       "      <td>-1.140721</td>\n",
       "      <td>0.278601</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.539367</td>\n",
       "      <td>0.769265</td>\n",
       "      <td>-0.707399</td>\n",
       "      <td>-0.341294</td>\n",
       "      <td>abortion</td>\n",
       "      <td>1.0</td>\n",
       "      <td>13</td>\n",
       "      <td>17.778846</td>\n",
       "      <td>46.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.010476</td>\n",
       "      <td>0.964721</td>\n",
       "      <td>1.305634</td>\n",
       "      <td>-0.034877</td>\n",
       "      <td>0.474277</td>\n",
       "      <td>-0.898328</td>\n",
       "      <td>-1.100798</td>\n",
       "      <td>-0.039020</td>\n",
       "      <td>0.823545</td>\n",
       "      <td>-0.465004</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.526546</td>\n",
       "      <td>-0.527923</td>\n",
       "      <td>-1.859096</td>\n",
       "      <td>-0.061935</td>\n",
       "      <td>energy,history,job-accomplishments</td>\n",
       "      <td>2.0</td>\n",
       "      <td>28</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>28.0</td>\n",
       "      <td>28.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1.140447</td>\n",
       "      <td>-0.794626</td>\n",
       "      <td>-0.289983</td>\n",
       "      <td>-1.464744</td>\n",
       "      <td>0.150040</td>\n",
       "      <td>-0.164812</td>\n",
       "      <td>-0.585933</td>\n",
       "      <td>1.033618</td>\n",
       "      <td>-0.524356</td>\n",
       "      <td>0.459435</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.965426</td>\n",
       "      <td>-1.898334</td>\n",
       "      <td>0.173119</td>\n",
       "      <td>-0.242052</td>\n",
       "      <td>foreign-policy</td>\n",
       "      <td>3.0</td>\n",
       "      <td>21</td>\n",
       "      <td>24.240000</td>\n",
       "      <td>515.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.584749</td>\n",
       "      <td>0.532029</td>\n",
       "      <td>0.670361</td>\n",
       "      <td>0.512556</td>\n",
       "      <td>0.771463</td>\n",
       "      <td>-0.094614</td>\n",
       "      <td>0.287700</td>\n",
       "      <td>-0.281629</td>\n",
       "      <td>-0.467286</td>\n",
       "      <td>-0.025476</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.029822</td>\n",
       "      <td>1.412071</td>\n",
       "      <td>0.784867</td>\n",
       "      <td>-0.546644</td>\n",
       "      <td>health-care</td>\n",
       "      <td>1.0</td>\n",
       "      <td>14</td>\n",
       "      <td>20.617577</td>\n",
       "      <td>56.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.061722</td>\n",
       "      <td>0.049816</td>\n",
       "      <td>-0.310454</td>\n",
       "      <td>0.585992</td>\n",
       "      <td>1.065714</td>\n",
       "      <td>-0.015555</td>\n",
       "      <td>-0.189076</td>\n",
       "      <td>-0.759746</td>\n",
       "      <td>0.832176</td>\n",
       "      <td>-0.242550</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.096825</td>\n",
       "      <td>0.597541</td>\n",
       "      <td>-0.825019</td>\n",
       "      <td>-0.632962</td>\n",
       "      <td>economy,jobs</td>\n",
       "      <td>2.0</td>\n",
       "      <td>12</td>\n",
       "      <td>19.948529</td>\n",
       "      <td>41.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1030 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2         3         4         5         6  \\\n",
       "0 -0.819478  0.553745  0.281095  0.483959  0.425502  0.377212  1.252342   \n",
       "1  0.010476  0.964721  1.305634 -0.034877  0.474277 -0.898328 -1.100798   \n",
       "2 -1.140447 -0.794626 -0.289983 -1.464744  0.150040 -0.164812 -0.585933   \n",
       "3  0.584749  0.532029  0.670361  0.512556  0.771463 -0.094614  0.287700   \n",
       "4  0.061722  0.049816 -0.310454  0.585992  1.065714 -0.015555 -0.189076   \n",
       "\n",
       "          7         8         9  ...      1020      1021      1022      1023  \\\n",
       "0 -0.328106 -1.140721  0.278601  ... -0.539367  0.769265 -0.707399 -0.341294   \n",
       "1 -0.039020  0.823545 -0.465004  ... -0.526546 -0.527923 -1.859096 -0.061935   \n",
       "2  1.033618 -0.524356  0.459435  ... -0.965426 -1.898334  0.173119 -0.242052   \n",
       "3 -0.281629 -0.467286 -0.025476  ... -0.029822  1.412071  0.784867 -0.546644   \n",
       "4 -0.759746  0.832176 -0.242550  ... -0.096825  0.597541 -0.825019 -0.632962   \n",
       "\n",
       "                             Text_Tag  Labels  description_word_len  \\\n",
       "0                            abortion     1.0                    13   \n",
       "1  energy,history,job-accomplishments     2.0                    28   \n",
       "2                      foreign-policy     3.0                    21   \n",
       "3                         health-care     1.0                    14   \n",
       "4                        economy,jobs     2.0                    12   \n",
       "\n",
       "        mean    max   min  \n",
       "0  17.778846   46.0   3.0  \n",
       "1  28.000000   28.0  28.0  \n",
       "2  24.240000  515.0   6.0  \n",
       "3  20.617577   56.0   6.0  \n",
       "4  19.948529   41.0   7.0  \n",
       "\n",
       "[5 rows x 1030 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.DataFrame(sentence_embeddings)\n",
    "for i in [\"Text_Tag\",\"Labels\",\"description_word_len\",\"mean\",\"max\",\"min\"]:\n",
    "  data[i] = merge[i].values\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0eHpROJG1Bg5"
   },
   "outputs": [],
   "source": [
    "train = data[~data.Labels.isna()]\n",
    "test = data[data.Labels.isna()]\n",
    "test.drop(\"Labels\",axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 422
    },
    "colab_type": "code",
    "id": "3_xitcgU3y8n",
    "outputId": "da3b0523-f8d0-4169-daab-66a5b8f5f119"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>1020</th>\n",
       "      <th>1021</th>\n",
       "      <th>1022</th>\n",
       "      <th>1023</th>\n",
       "      <th>Text_Tag</th>\n",
       "      <th>Labels</th>\n",
       "      <th>description_word_len</th>\n",
       "      <th>mean</th>\n",
       "      <th>max</th>\n",
       "      <th>min</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.819478</td>\n",
       "      <td>0.553745</td>\n",
       "      <td>0.281095</td>\n",
       "      <td>0.483959</td>\n",
       "      <td>0.425502</td>\n",
       "      <td>0.377212</td>\n",
       "      <td>1.252342</td>\n",
       "      <td>-0.328106</td>\n",
       "      <td>-1.140721</td>\n",
       "      <td>0.278601</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.539367</td>\n",
       "      <td>0.769265</td>\n",
       "      <td>-0.707399</td>\n",
       "      <td>-0.341294</td>\n",
       "      <td>abortion</td>\n",
       "      <td>1.0</td>\n",
       "      <td>13</td>\n",
       "      <td>17.778846</td>\n",
       "      <td>46.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.010476</td>\n",
       "      <td>0.964721</td>\n",
       "      <td>1.305634</td>\n",
       "      <td>-0.034877</td>\n",
       "      <td>0.474277</td>\n",
       "      <td>-0.898328</td>\n",
       "      <td>-1.100798</td>\n",
       "      <td>-0.039020</td>\n",
       "      <td>0.823545</td>\n",
       "      <td>-0.465004</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.526546</td>\n",
       "      <td>-0.527923</td>\n",
       "      <td>-1.859096</td>\n",
       "      <td>-0.061935</td>\n",
       "      <td>energy,history,job-accomplishments</td>\n",
       "      <td>2.0</td>\n",
       "      <td>28</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>28.0</td>\n",
       "      <td>28.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1.140447</td>\n",
       "      <td>-0.794626</td>\n",
       "      <td>-0.289983</td>\n",
       "      <td>-1.464744</td>\n",
       "      <td>0.150040</td>\n",
       "      <td>-0.164812</td>\n",
       "      <td>-0.585933</td>\n",
       "      <td>1.033618</td>\n",
       "      <td>-0.524356</td>\n",
       "      <td>0.459435</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.965426</td>\n",
       "      <td>-1.898334</td>\n",
       "      <td>0.173119</td>\n",
       "      <td>-0.242052</td>\n",
       "      <td>foreign-policy</td>\n",
       "      <td>3.0</td>\n",
       "      <td>21</td>\n",
       "      <td>24.240000</td>\n",
       "      <td>515.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.584749</td>\n",
       "      <td>0.532029</td>\n",
       "      <td>0.670361</td>\n",
       "      <td>0.512556</td>\n",
       "      <td>0.771463</td>\n",
       "      <td>-0.094614</td>\n",
       "      <td>0.287700</td>\n",
       "      <td>-0.281629</td>\n",
       "      <td>-0.467286</td>\n",
       "      <td>-0.025476</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.029822</td>\n",
       "      <td>1.412071</td>\n",
       "      <td>0.784867</td>\n",
       "      <td>-0.546644</td>\n",
       "      <td>health-care</td>\n",
       "      <td>1.0</td>\n",
       "      <td>14</td>\n",
       "      <td>20.617577</td>\n",
       "      <td>56.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.061722</td>\n",
       "      <td>0.049816</td>\n",
       "      <td>-0.310454</td>\n",
       "      <td>0.585992</td>\n",
       "      <td>1.065714</td>\n",
       "      <td>-0.015555</td>\n",
       "      <td>-0.189076</td>\n",
       "      <td>-0.759746</td>\n",
       "      <td>0.832176</td>\n",
       "      <td>-0.242550</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.096825</td>\n",
       "      <td>0.597541</td>\n",
       "      <td>-0.825019</td>\n",
       "      <td>-0.632962</td>\n",
       "      <td>economy,jobs</td>\n",
       "      <td>2.0</td>\n",
       "      <td>12</td>\n",
       "      <td>19.948529</td>\n",
       "      <td>41.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1030 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2         3         4         5         6  \\\n",
       "0 -0.819478  0.553745  0.281095  0.483959  0.425502  0.377212  1.252342   \n",
       "1  0.010476  0.964721  1.305634 -0.034877  0.474277 -0.898328 -1.100798   \n",
       "2 -1.140447 -0.794626 -0.289983 -1.464744  0.150040 -0.164812 -0.585933   \n",
       "3  0.584749  0.532029  0.670361  0.512556  0.771463 -0.094614  0.287700   \n",
       "4  0.061722  0.049816 -0.310454  0.585992  1.065714 -0.015555 -0.189076   \n",
       "\n",
       "          7         8         9  ...      1020      1021      1022      1023  \\\n",
       "0 -0.328106 -1.140721  0.278601  ... -0.539367  0.769265 -0.707399 -0.341294   \n",
       "1 -0.039020  0.823545 -0.465004  ... -0.526546 -0.527923 -1.859096 -0.061935   \n",
       "2  1.033618 -0.524356  0.459435  ... -0.965426 -1.898334  0.173119 -0.242052   \n",
       "3 -0.281629 -0.467286 -0.025476  ... -0.029822  1.412071  0.784867 -0.546644   \n",
       "4 -0.759746  0.832176 -0.242550  ... -0.096825  0.597541 -0.825019 -0.632962   \n",
       "\n",
       "                             Text_Tag  Labels  description_word_len  \\\n",
       "0                            abortion     1.0                    13   \n",
       "1  energy,history,job-accomplishments     2.0                    28   \n",
       "2                      foreign-policy     3.0                    21   \n",
       "3                         health-care     1.0                    14   \n",
       "4                        economy,jobs     2.0                    12   \n",
       "\n",
       "        mean    max   min  \n",
       "0  17.778846   46.0   3.0  \n",
       "1  28.000000   28.0  28.0  \n",
       "2  24.240000  515.0   6.0  \n",
       "3  20.617577   56.0   6.0  \n",
       "4  19.948529   41.0   7.0  \n",
       "\n",
       "[5 rows x 1030 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['Text_Tag'] = train['Text_Tag'].fillna(\"\").apply(lambda x : ' '.join(x.split(',')))\n",
    "test['Text_Tag'] = test['Text_Tag'].fillna(\"\").apply(lambda x : ' '.join(x.split(',')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "jY7Flv5_tLZ1",
    "outputId": "981e4075-fddd-40bb-bc02-bfe9462f19a1"
   },
   "outputs": [],
   "source": [
    "X = train.drop([\"Labels\"],axis=1)\n",
    "Y = train[['Labels']]\n",
    "\n",
    "params = {\n",
    "    \"od_type\":\"Iter\",\n",
    "    \"od_wait\":150,\n",
    "    \"iterations\":25000,\n",
    "    'learning_rate':0.02,\n",
    "    \"eval_metric\":\"Accuracy\",\n",
    "    \"task_type\":\"GPU\",\n",
    "    \"boosting_type\":\"Plain\"\n",
    "}\n",
    "\n",
    "best_score = np.inf\n",
    "scores = []\n",
    "\n",
    "folds_large = KFold(n_splits=5,shuffle=True,random_state=1250)\n",
    "\n",
    "for train_idx , test_idx in folds_large.split(X,Y):\n",
    "  train_set = (X.iloc[train_idx],Y.iloc[train_idx])\n",
    "  test_set = (X.iloc[test_idx],Y.iloc[test_idx])\n",
    "\n",
    "  model = CatBoostClassifier(**params)\n",
    "  model.fit(*train_set,\n",
    "            cat_features = [\"Text_Tag\"],\n",
    "            eval_set=[test_set],early_stopping_rounds=500,verbose=200)\n",
    "\n",
    "  score = log_loss(test_set[1].values,model.predict_proba(test_set[0]))\n",
    "  print(score)\n",
    "  scores.append(score)\n",
    "  \n",
    "\n",
    "  if score < best_score:\n",
    "    best_score = score\n",
    "    best_model = model\n",
    "\n",
    "  print(\"---\"*50)\n",
    "\n",
    "print(f\"Mean Score : {np.array(scores).mean()}\")\n",
    "print(f\"Min Score : {np.array(scores).min()}\")\n",
    "print(f\"Max Score : {np.array(scores).max()}\")\n",
    "\n",
    "plt.plot(scores)\n",
    "plt.plot(np.arange(len(scores)),[np.array(scores).mean()]*len(scores),)\n",
    "plt.show()\n",
    "\n",
    "model_large = best_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 402
    },
    "colab_type": "code",
    "id": "ea64m-cclHJK",
    "outputId": "e90557b2-acb8-416a-ca0f-787ffe5a5a4f"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>32</td>\n",
       "      <td>0.904796</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>747</th>\n",
       "      <td>747</td>\n",
       "      <td>0.877003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1025</th>\n",
       "      <td>description_word_len</td>\n",
       "      <td>0.785462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>987</th>\n",
       "      <td>987</td>\n",
       "      <td>0.738166</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>601</th>\n",
       "      <td>601</td>\n",
       "      <td>0.634172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118</th>\n",
       "      <td>118</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>428</th>\n",
       "      <td>428</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>437</th>\n",
       "      <td>437</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>611</th>\n",
       "      <td>611</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>774</th>\n",
       "      <td>774</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1029 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                         0         1\n",
       "32                      32  0.904796\n",
       "747                    747  0.877003\n",
       "1025  description_word_len  0.785462\n",
       "987                    987  0.738166\n",
       "601                    601  0.634172\n",
       "...                    ...       ...\n",
       "118                    118  0.000000\n",
       "428                    428  0.000000\n",
       "437                    437  0.000000\n",
       "611                    611  0.000000\n",
       "774                    774  0.000000\n",
       "\n",
       "[1029 rows x 2 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.set_option(\"display.max_colwidth\",100)\n",
    "pd.set_option(\"display.max_rows\",150)\n",
    "m = pd.DataFrame(list(zip(X.columns,model_large.feature_importances_))).sort_values(1,ascending=False)\n",
    "m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission = pd.DataFrame(model_large.predict_proba(test))\n",
    "submission.columns = [f'Class_{i}' for i in submission.columns]\n",
    "submission.to_csv(\"Sub_v0.4.csv\",index=False)\n",
    "submission"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
